{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import os\n",
    "import json\n",
    "import tabulate\n",
    "from collections import Counter\n",
    "from IPython.display import HTML, display"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load wordstat json logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading 54 files...0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, \n",
      "Finished loading files\n"
     ]
    }
   ],
   "source": [
    "models_dir = '~/ParlAI/data/controllable_dialogue/wordstat_files'  # Enter the path to your wordstat_files directory here\n",
    "wordstat_files = [fname for fname in os.listdir(models_dir) if 'wordstats.json' in fname]\n",
    "mf2data = {} # master dict mapping model file name to its data dict\n",
    "\n",
    "print('Loading %i files...' % len(wordstat_files), end='')\n",
    "for idx, json_file in enumerate(sorted(wordstat_files)):\n",
    "    mf = json_file[:json_file.index('.wordstats.json')]\n",
    "    print('%i, ' % idx, end='')\n",
    "    with open(os.path.join(models_dir, json_file), \"r\") as f:\n",
    "        data = json.load(f)\n",
    "    mf2data[mf] = data\n",
    "print('\\nFinished loading files')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Make table of automatic metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tbody>\n",
       "<tr><td style=\"text-align: left;\">                                                                                                 model name                                                                                                 </td><td style=\"text-align: left;\">extrep_2gram</td><td style=\"text-align: left;\">extrep_nonstopword</td><td style=\"text-align: left;\">intrep_2gram</td><td style=\"text-align: left;\">intrep_nonstopword</td><td style=\"text-align: left;\">partnerrep_2gram</td><td style=\"text-align: left;\">avg_nidf</td><td style=\"text-align: left;\">lastuttsim</td><td style=\"text-align: left;\">question</td></tr>\n",
       "<tr><td style=\"text-align: left;\">                                                                                                goldresponse                                                                                                </td><td style=\"text-align: left;\">   4.65%    </td><td style=\"text-align: left;\">      9.62%       </td><td style=\"text-align: left;\">   0.38%    </td><td style=\"text-align: left;\">      0.97%       </td><td style=\"text-align: left;\">     5.10%      </td><td style=\"text-align: left;\"> 0.2119 </td><td style=\"text-align: left;\">  0.1691  </td><td style=\"text-align: left;\"> 28.80% </td></tr>\n",
       "<tr><td style=\"text-align: left;\">                                                                            convai2_finetuned_baseline.valid.usemodelreply.beam1                                                                            </td><td style=\"text-align: left;\">   35.88%   </td><td style=\"text-align: left;\">      36.31%      </td><td style=\"text-align: left;\">   8.08%    </td><td style=\"text-align: left;\">      10.59%      </td><td style=\"text-align: left;\">     12.20%     </td><td style=\"text-align: left;\"> 0.1688 </td><td style=\"text-align: left;\">  0.1850  </td><td style=\"text-align: left;\"> 6.46%  </td></tr>\n",
       "<tr><td style=\"text-align: left;\">                                                                    convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10                                                                    </td><td style=\"text-align: left;\">   46.85%   </td><td style=\"text-align: left;\">      44.15%      </td><td style=\"text-align: left;\">   0.32%    </td><td style=\"text-align: left;\">      0.61%       </td><td style=\"text-align: left;\">     12.90%     </td><td style=\"text-align: left;\"> 0.1662 </td><td style=\"text-align: left;\">  0.0957  </td><td style=\"text-align: left;\"> 80.87% </td></tr>\n",
       "<tr><td style=\"text-align: left;\">                                                      convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-0.5                                                      </td><td style=\"text-align: left;\">   19.70%   </td><td style=\"text-align: left;\">      16.85%      </td><td style=\"text-align: left;\">   0.26%    </td><td style=\"text-align: left;\">      0.62%       </td><td style=\"text-align: left;\">     11.93%     </td><td style=\"text-align: left;\"> 0.1730 </td><td style=\"text-align: left;\">  0.1348  </td><td style=\"text-align: left;\"> 73.04% </td></tr>\n",
       "<tr><td style=\"text-align: left;\">                                                     convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-1.25                                                      </td><td style=\"text-align: left;\">   4.62%    </td><td style=\"text-align: left;\">      4.79%       </td><td style=\"text-align: left;\">   0.40%    </td><td style=\"text-align: left;\">      0.89%       </td><td style=\"text-align: left;\">     10.61%     </td><td style=\"text-align: left;\"> 0.1763 </td><td style=\"text-align: left;\">  0.1504  </td><td style=\"text-align: left;\"> 61.22% </td></tr>\n",
       "<tr><td style=\"text-align: left;\">                                                      convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-3.5                                                      </td><td style=\"text-align: left;\">   0.75%    </td><td style=\"text-align: left;\">      4.61%       </td><td style=\"text-align: left;\">   0.47%    </td><td style=\"text-align: left;\">      0.94%       </td><td style=\"text-align: left;\">     9.89%      </td><td style=\"text-align: left;\"> 0.1771 </td><td style=\"text-align: left;\">  0.1681  </td><td style=\"text-align: left;\"> 48.89% </td></tr>\n",
       "<tr><td style=\"text-align: left;\">                                                     convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-1e+20                                                     </td><td style=\"text-align: left;\">   0.00%    </td><td style=\"text-align: left;\">      4.74%       </td><td style=\"text-align: left;\">   0.51%    </td><td style=\"text-align: left;\">      1.05%       </td><td style=\"text-align: left;\">     9.56%      </td><td style=\"text-align: left;\"> 0.1780 </td><td style=\"text-align: left;\">  0.1711  </td><td style=\"text-align: left;\"> 45.98% </td></tr>\n",
       "<tr><td style=\"text-align: left;\">                             convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20                             </td><td style=\"text-align: left;\">   0.73%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">   0.17%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">     9.55%      </td><td style=\"text-align: left;\"> 0.1766 </td><td style=\"text-align: left;\">  0.1676  </td><td style=\"text-align: left;\"> 49.98% </td></tr>\n",
       "<tr><td style=\"text-align: left;\">                    control_questionb11e10.valid.usemodelreply.beam20.beamminnbest10.setcontrols:question0.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20                    </td><td style=\"text-align: left;\">   0.06%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">   0.19%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">     9.20%      </td><td style=\"text-align: left;\"> 0.1871 </td><td style=\"text-align: left;\">  0.1753  </td><td style=\"text-align: left;\"> 2.01%  </td></tr>\n",
       "<tr><td style=\"text-align: left;\">                    control_questionb11e10.valid.usemodelreply.beam20.beamminnbest10.setcontrols:question1.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20                    </td><td style=\"text-align: left;\">   0.09%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">   0.19%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">     8.66%      </td><td style=\"text-align: left;\"> 0.1844 </td><td style=\"text-align: left;\">  0.1722  </td><td style=\"text-align: left;\"> 17.33% </td></tr>\n",
       "<tr><td style=\"text-align: left;\">                    control_questionb11e10.valid.usemodelreply.beam20.beamminnbest10.setcontrols:question4.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20                    </td><td style=\"text-align: left;\">   0.40%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">   0.25%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">     8.53%      </td><td style=\"text-align: left;\"> 0.1794 </td><td style=\"text-align: left;\">  0.1713  </td><td style=\"text-align: left;\"> 48.88% </td></tr>\n",
       "<tr><td style=\"text-align: left;\">                    control_questionb11e10.valid.usemodelreply.beam20.beamminnbest10.setcontrols:question7.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20                    </td><td style=\"text-align: left;\">   0.80%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">   0.17%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">     8.48%      </td><td style=\"text-align: left;\"> 0.1771 </td><td style=\"text-align: left;\">  0.1724  </td><td style=\"text-align: left;\"> 65.65% </td></tr>\n",
       "<tr><td style=\"text-align: left;\">                   control_questionb11e10.valid.usemodelreply.beam20.beamminnbest10.setcontrols:question10.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20                    </td><td style=\"text-align: left;\">   1.27%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">   0.16%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">     8.48%      </td><td style=\"text-align: left;\"> 0.1761 </td><td style=\"text-align: left;\">  0.1728  </td><td style=\"text-align: left;\"> 79.67% </td></tr>\n",
       "<tr><td style=\"text-align: left;\">            control_questionb11e10.valid.usemodelreply.beam20.beamminnbest10.setcontrols:question10.beamreorder_best_extrep2gram_qn.WDfeatures:extrep_nonstopword-1e+20_intrep_nonstopword-1e+20            </td><td style=\"text-align: left;\">   7.64%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">   0.03%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">     10.76%     </td><td style=\"text-align: left;\"> 0.1701 </td><td style=\"text-align: left;\">  0.1651  </td><td style=\"text-align: left;\"> 99.54% </td></tr>\n",
       "<tr><td style=\"text-align: left;\">                    control_avgnidf10b10e.valid.usemodelreply.beam20.beamminnbest10.setcontrols:avg_nidf0.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20                     </td><td style=\"text-align: left;\">   0.60%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">   0.20%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">     9.05%      </td><td style=\"text-align: left;\"> 0.1478 </td><td style=\"text-align: left;\">  0.1522  </td><td style=\"text-align: left;\"> 48.75% </td></tr>\n",
       "<tr><td style=\"text-align: left;\">                    control_avgnidf10b10e.valid.usemodelreply.beam20.beamminnbest10.setcontrols:avg_nidf2.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20                     </td><td style=\"text-align: left;\">   0.28%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">   0.10%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">     8.37%      </td><td style=\"text-align: left;\"> 0.1772 </td><td style=\"text-align: left;\">  0.1833  </td><td style=\"text-align: left;\"> 50.57% </td></tr>\n",
       "<tr><td style=\"text-align: left;\">                    control_avgnidf10b10e.valid.usemodelreply.beam20.beamminnbest10.setcontrols:avg_nidf4.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20                     </td><td style=\"text-align: left;\">   0.12%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">   0.08%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">     7.90%      </td><td style=\"text-align: left;\"> 0.1921 </td><td style=\"text-align: left;\">  0.1877  </td><td style=\"text-align: left;\"> 29.46% </td></tr>\n",
       "<tr><td style=\"text-align: left;\">                    control_avgnidf10b10e.valid.usemodelreply.beam20.beamminnbest10.setcontrols:avg_nidf7.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20                     </td><td style=\"text-align: left;\">   0.02%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">   0.14%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">     8.17%      </td><td style=\"text-align: left;\"> 0.2156 </td><td style=\"text-align: left;\">  0.1955  </td><td style=\"text-align: left;\"> 16.51% </td></tr>\n",
       "<tr><td style=\"text-align: left;\">                    control_avgnidf10b10e.valid.usemodelreply.beam20.beamminnbest10.setcontrols:avg_nidf9.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20                     </td><td style=\"text-align: left;\">   0.01%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">   0.11%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">     8.01%      </td><td style=\"text-align: left;\"> 0.2462 </td><td style=\"text-align: left;\">  0.1990  </td><td style=\"text-align: left;\"> 8.50%  </td></tr>\n",
       "<tr><td style=\"text-align: left;\">                        convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20_nidf-10.0                        </td><td style=\"text-align: left;\">   0.14%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">   10.59%   </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">     8.70%      </td><td style=\"text-align: left;\"> 0.1107 </td><td style=\"text-align: left;\">  0.0994  </td><td style=\"text-align: left;\"> 33.55% </td></tr>\n",
       "<tr><td style=\"text-align: left;\">                        convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20_nidf-4.0                         </td><td style=\"text-align: left;\">   0.65%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">   1.98%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">     9.95%      </td><td style=\"text-align: left;\"> 0.1501 </td><td style=\"text-align: left;\">  0.1398  </td><td style=\"text-align: left;\"> 44.92% </td></tr>\n",
       "<tr><td style=\"text-align: left;\">                         convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20_nidf4.0                         </td><td style=\"text-align: left;\">   0.15%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">   0.19%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">     7.54%      </td><td style=\"text-align: left;\"> 0.2121 </td><td style=\"text-align: left;\">  0.1972  </td><td style=\"text-align: left;\"> 45.53% </td></tr>\n",
       "<tr><td style=\"text-align: left;\">                         convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20_nidf6.0                         </td><td style=\"text-align: left;\">   0.07%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">   0.13%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">     6.50%      </td><td style=\"text-align: left;\"> 0.2546 </td><td style=\"text-align: left;\">  0.2040  </td><td style=\"text-align: left;\"> 39.37% </td></tr>\n",
       "<tr><td style=\"text-align: left;\">                         convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20_nidf8.0                         </td><td style=\"text-align: left;\">   0.01%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">   0.10%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">     3.40%      </td><td style=\"text-align: left;\"> 0.4035 </td><td style=\"text-align: left;\">  0.1436  </td><td style=\"text-align: left;\"> 26.68% </td></tr>\n",
       "<tr><td style=\"text-align: left;\">convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_2gram-1e+20_intrep_nonstopword-1e+20_lastuttsim-10.0_partnerrep_2gram-1e+20</td><td style=\"text-align: left;\">   0.13%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">   0.00%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">     0.00%      </td><td style=\"text-align: left;\"> 0.1914 </td><td style=\"text-align: left;\"> -0.0921  </td><td style=\"text-align: left;\"> 25.71% </td></tr>\n",
       "<tr><td style=\"text-align: left;\"> convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_2gram-1e+20_intrep_nonstopword-1e+20_lastuttsim0.0_partnerrep_2gram-1e+20 </td><td style=\"text-align: left;\">   0.24%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">   0.00%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">     0.00%      </td><td style=\"text-align: left;\"> 0.1785 </td><td style=\"text-align: left;\">  0.1414  </td><td style=\"text-align: left;\"> 44.55% </td></tr>\n",
       "<tr><td style=\"text-align: left;\"> convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_2gram-1e+20_intrep_nonstopword-1e+20_lastuttsim5.0_partnerrep_2gram-1e+20 </td><td style=\"text-align: left;\">   0.15%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">   0.00%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">     0.00%      </td><td style=\"text-align: left;\"> 0.1973 </td><td style=\"text-align: left;\">  0.4360  </td><td style=\"text-align: left;\"> 39.78% </td></tr>\n",
       "<tr><td style=\"text-align: left;\">convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_2gram-1e+20_intrep_nonstopword-1e+20_lastuttsim10.0_partnerrep_2gram-1e+20 </td><td style=\"text-align: left;\">   0.05%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">   0.00%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">     0.00%      </td><td style=\"text-align: left;\"> 0.2535 </td><td style=\"text-align: left;\">  0.6653  </td><td style=\"text-align: left;\"> 27.56% </td></tr>\n",
       "<tr><td style=\"text-align: left;\">convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_2gram-1e+20_intrep_nonstopword-1e+20_lastuttsim13.0_partnerrep_2gram-1e+20 </td><td style=\"text-align: left;\">   0.02%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">   0.00%    </td><td style=\"text-align: left;\">      0.00%       </td><td style=\"text-align: left;\">     0.00%      </td><td style=\"text-align: left;\"> 0.2999 </td><td style=\"text-align: left;\">  0.7251  </td><td style=\"text-align: left;\"> 20.47% </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# This cell makes Table 6 from the paper\n",
    "\n",
    "columns = [\n",
    "    'extrep_2gram',\n",
    "    'extrep_nonstopword',\n",
    "    'intrep_2gram',\n",
    "    'intrep_nonstopword',\n",
    "    'partnerrep_2gram',\n",
    "    'avg_nidf',\n",
    "    'lastuttsim',\n",
    "    'question',\n",
    "]\n",
    "\n",
    "header_row = ['model name'] + columns\n",
    "\n",
    "rows = [\n",
    "    # gold data and baselines\n",
    "    'goldresponse',\n",
    "    'convai2_finetuned_baseline.valid.usemodelreply.beam1',\n",
    "    'convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10',\n",
    "\n",
    "    # repetition control (WD)\n",
    "    'convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-0.5',\n",
    "    'convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-1.25',\n",
    "    'convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-3.5',\n",
    "    'convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-1e+20',\n",
    "    'convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20',\n",
    "    \n",
    "    # question control (CT)\n",
    "    'control_questionb11e10.valid.usemodelreply.beam20.beamminnbest10.setcontrols:question0.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20',\n",
    "    'control_questionb11e10.valid.usemodelreply.beam20.beamminnbest10.setcontrols:question1.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20',\n",
    "    'control_questionb11e10.valid.usemodelreply.beam20.beamminnbest10.setcontrols:question4.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20',\n",
    "    'control_questionb11e10.valid.usemodelreply.beam20.beamminnbest10.setcontrols:question7.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20',\n",
    "    'control_questionb11e10.valid.usemodelreply.beam20.beamminnbest10.setcontrols:question10.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20',\n",
    "    'control_questionb11e10.valid.usemodelreply.beam20.beamminnbest10.setcontrols:question10.beamreorder_best_extrep2gram_qn.WDfeatures:extrep_nonstopword-1e+20_intrep_nonstopword-1e+20',\n",
    "\n",
    "    # specificity control (CT)\n",
    "    'control_avgnidf10b10e.valid.usemodelreply.beam20.beamminnbest10.setcontrols:avg_nidf0.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20',\n",
    "    'control_avgnidf10b10e.valid.usemodelreply.beam20.beamminnbest10.setcontrols:avg_nidf2.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20',\n",
    "    'control_avgnidf10b10e.valid.usemodelreply.beam20.beamminnbest10.setcontrols:avg_nidf4.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20',\n",
    "    'control_avgnidf10b10e.valid.usemodelreply.beam20.beamminnbest10.setcontrols:avg_nidf7.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20',\n",
    "    'control_avgnidf10b10e.valid.usemodelreply.beam20.beamminnbest10.setcontrols:avg_nidf9.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20',\n",
    "\n",
    "    # specificity control (WD)\n",
    "    'convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20_nidf-10.0',\n",
    "    'convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20_nidf-4.0',\n",
    "    'convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20_nidf4.0',\n",
    "    'convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20_nidf6.0',\n",
    "    'convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_nonstopword-1e+20_nidf8.0',\n",
    "    \n",
    "    # response-related control (WD)\n",
    "    'convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_2gram-1e+20_intrep_nonstopword-1e+20_lastuttsim-10.0_partnerrep_2gram-1e+20',\n",
    "    'convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_2gram-1e+20_intrep_nonstopword-1e+20_lastuttsim0.0_partnerrep_2gram-1e+20',\n",
    "    'convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_2gram-1e+20_intrep_nonstopword-1e+20_lastuttsim5.0_partnerrep_2gram-1e+20',\n",
    "    'convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_2gram-1e+20_intrep_nonstopword-1e+20_lastuttsim10.0_partnerrep_2gram-1e+20',\n",
    "    'convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10.WDfeatures:extrep_2gram-3.5_extrep_nonstopword-1e+20_intrep_2gram-1e+20_intrep_nonstopword-1e+20_lastuttsim13.0_partnerrep_2gram-1e+20',\n",
    "\n",
    "]\n",
    "\n",
    "def mean(l):\n",
    "    return sum(l)/len(l)\n",
    "\n",
    "def model2row(mf, data):\n",
    "    \"\"\"Given the data from a json file, make a row of data for the table\"\"\"\n",
    "    row = [mf]\n",
    "    for attr in columns:\n",
    "        sent_attrs = data['sent_attrs']\n",
    "        if attr in sent_attrs:\n",
    "            attr_mean = mean(sent_attrs[attr])\n",
    "            if attr in ['avg_nidf', 'lastuttsim']:\n",
    "                row.append(\"%.4f\" % (attr_mean))\n",
    "            else:\n",
    "                row.append(\"%.2f%%\" % (attr_mean*100))\n",
    "        else:\n",
    "            row.append('')\n",
    "    return row\n",
    "\n",
    "# Build table\n",
    "table = [header_row] \n",
    "for mf in rows:\n",
    "    data = mf2data[mf]\n",
    "    table.append(model2row(mf, data))\n",
    "html = HTML(tabulate.tabulate(table, tablefmt='html', stralign='center'))\n",
    "html.data = html.data.replace(\"text-align: center;\", \"text-align: left;\") # fix left-alignment \n",
    "display(html)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Show predictions of a model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "% of utterances that are unique: 14.77% (1152/7801)\n",
      "\n",
      "COUNT   UTTERANCE\n",
      " 2945   what city are you from\n",
      " 1245   what do you do for living\n",
      "  205   i am good how are you\n",
      "  190   do you have any pets\n",
      "  104   what kind of music do you like\n",
      "   97   hi how are you today\n",
      "   80   do you have any hobbies\n",
      "   71   i am great how are you\n",
      "   51   no i do not do you\n",
      "   44   do you play any instruments\n",
      "   42   what kind of music do you play\n",
      "   41   hello how are you today\n",
      "   34   i am in cali\n",
      "   31   i am good thanks for asking\n",
      "   30   i am stay at home mom\n",
      "   29   that sounds like lot of fun\n",
      "   28   i am well how are you\n",
      "   27   i am doing well how are you\n",
      "   27   what kind of work do you do\n",
      "   26   what do you do for work\n",
      "   25   what is your favorite food\n",
      "   23   what is your favorite color\n",
      "   22   i do not have any pets\n",
      "   22   i am from los angeles\n",
      "   21   what languages do you speak\n",
      "   19   what kind of dog do you have\n",
      "   19   what kind of dogs do you have\n",
      "   19   what kind of car do you drive\n",
      "   17   i am in los angeles\n",
      "   16   i m doing well how are you\n",
      "   16   i am doing well how about you\n",
      "   16   how are you doing today\n",
      "   15   i do not do you\n",
      "   15   i am sorry to hear that\n",
      "   15   i am from united states\n",
      "   14   how many kids do you have\n",
      "   14   i am in 3rd grade\n",
      "   14   i like all kinds of music\n",
      "   13   what are you going to school for\n",
      "   13   i was forced to marry when i was young\n",
      "   12   what kind of food do you like\n",
      "   12   what color is your hair\n",
      "   12   do you have any children\n",
      "   12   do you have any kids\n",
      "   12   what is your favorite food mine is mexican\n",
      "   11   i live in los angeles\n",
      "   11   what kind of music do you listen to\n",
      "   10   my ex cheated on me and left me for lawyer\n",
      "   10   i m good how are you\n",
      "    9   what instrument do you play\n",
      "    9   hi how are you doing\n",
      "    9   i used to be painter but now i am retired\n",
      "    9   what do you do for fun\n",
      "    9   i like to listen to music\n",
      "    9   what is your favorite color mine is red\n",
      "    8   i m doing well how about you\n",
      "    8   not too bad how about you\n",
      "    8   i m sorry to hear that\n",
      "    8   no i don t do you\n",
      "    8   i like to spend time with my family\n",
      "    8   i love red reminds me of summer time\n",
      "    7   sorry to hear that what do you do for living\n",
      "    7   good morning how are you\n",
      "    7   what is your favorite thing to buy\n",
      "    7   i live in alabama where do you live\n",
      "    6   what kind of car do you have\n",
      "    6   what is your favorite color mine is yellow\n",
      "    6   do you have any siblings\n",
      "    6   sure what do you do for living\n",
      "    6   no i have not do you\n",
      "    6   no i am stay at home mom\n",
      "    6   i work as construction worker\n",
      "    6   i love pizza what about you\n",
      "    6   do you have any family\n",
      "    6   do you play any sports\n",
      "    6   my wife left me and took care of my children\n",
      "    6   what is your favorite season mine is winter\n",
      "    6   i was raised on horse farm\n",
      "    5   i walk dogs for living\n",
      "    5   thank you what do you do for living\n",
      "    5   that sounds like plan\n",
      "    5   i like all kinds what about you\n",
      "    5   what do you do in your spare time\n",
      "    5   hi there how are you\n",
      "    5   i am pregnant with my first child\n",
      "    5   yes it is what do you do for living\n",
      "    5   yes i do what do you do for living\n",
      "    5   my dad taught me everything i know he taught me everything i know\n",
      "    5   i do not have any pets do you\n",
      "    5   happy birthday what do you like to do for fun\n",
      "    5   i grew up on farm\n",
      "    5   i like to cook and cook\n",
      "    5   i am country music singer\n",
      "    5   what is your favorite book\n",
      "    4   pretty good thanks and you\n",
      "    4   what kind of games do you play\n",
      "    4   sorry to hear that what happened\n",
      "    4   no i don t have any kids\n",
      "    4   i play piano and sing folk music\n",
      "    4   yes i do what do you do\n"
     ]
    }
   ],
   "source": [
    "mf = 'convai2_finetuned_baseline.valid.usemodelreply.beam20.beamminnbest10' # beam search baseline\n",
    "num_show = 100  # Show the top 100 most common utterances\n",
    "\n",
    "def show_preds(mf, num_show=None):\n",
    "    counter = Counter()\n",
    "    preds = mf2data[mf]['word_statistics']['pred_list'] # this is the normalized version; use pure_pred_list for unnormalized\n",
    "    counter.update(preds)\n",
    "    num_unique = len([p for p,count in counter.items() if count==1])\n",
    "    print(\"%% of utterances that are unique: %.2f%% (%i/%i)\\n\" % (num_unique*100/sum(counter.values()), num_unique, sum(counter.values())))\n",
    "    print(\"COUNT   UTTERANCE\")\n",
    "    for p, count in counter.most_common(num_show):\n",
    "        print(\"%5i   %s\" % (count, p))\n",
    "\n",
    "show_preds(mf, num_show)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
